package com.jorado.search.solr.client;

import com.jorado.core.RTimer;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.*;
import java.io.*;
import java.net.*;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.zip.GZIPInputStream;
import java.util.zip.Inflater;
import java.util.zip.InflaterInputStream;

public class SolrPostTool {
    private static final String DEFAULT_POST_HOST = "localhost";
    private static final String DEFAULT_POST_PORT = "8983";
    private static final String VERSION_OF_THIS_TOOL = "5.0.0";
    private static final String DEFAULT_COMMIT = "yes";
    private static final String DEFAULT_OPTIMIZE = "no";
    private static final String DEFAULT_OUT = "no";
    private static final String DEFAULT_AUTO = "no";
    private static final String DEFAULT_RECURSIVE = "0";
    private static final int DEFAULT_WEB_DELAY = 10;
    private static final int MAX_WEB_DEPTH = 10;
    private static final String DEFAULT_CONTENT_TYPE = "application/xml";
    private static final String DEFAULT_FILE_TYPES = "xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log";
    private static final String BASIC_AUTH = "basicauth";
    static final String DATA_MODE_FILES = "files";
    static final String DATA_MODE_ARGS = "args";
    static final String DATA_MODE_STDIN = "stdin";
    static final String DATA_MODE_WEB = "web";
    static final String DEFAULT_DATA_MODE = "files";
    boolean auto;
    int recursive;
    int delay;
    String fileTypes;
    URL solrUrl;
    OutputStream out;
    String type;
    String format;
    String mode;
    boolean commit;
    boolean optimize;
    String[] args;
    private int currentDepth;
    static HashMap<String, String> mimeMap;
    FileFilter fileFilter;
    List<LinkedHashSet<URL>> backlog;
    Set<URL> visited;
    static final Set<String> DATA_MODES;
    static final String USAGE_STRING_SHORT = "Usage: java [SystemProperties] -jar post.jar [-h|-] [<file|folder|url|arg> [<file|folder|url|arg>...]]";
    boolean mockMode;
    PageFetcher pageFetcher;

    public static void main(final String[] args) {
        info("PostTool version 5.0.0");
        if (0 < args.length && ("-help".equals(args[0]) || "--help".equals(args[0]) || "-h".equals(args[0]))) {
            usage();
        } else {
            final SolrPostTool t = parseArgsAndInit(args);
            t.execute();
        }
    }

    public void execute() {
        final RTimer timer = new RTimer();
        if ("files".equals(this.mode) && this.args.length > 0) {
            this.doFilesMode();
        } else if ("args".equals(this.mode) && this.args.length > 0) {
            this.doArgsMode();
        } else if ("web".equals(this.mode) && this.args.length > 0) {
            this.doWebMode();
        } else {
            if (!"stdin".equals(this.mode)) {
                usageShort();
                return;
            }
            this.doStdinMode();
        }
        if (this.commit) {
            this.commit();
        }
        if (this.optimize) {
            this.optimize();
        }
        this.displayTiming((long) timer.getTime());
    }

    private void displayTiming(final long millis) {
        final SimpleDateFormat df = new SimpleDateFormat("H:mm:ss.SSS", Locale.getDefault());
        df.setTimeZone(TimeZone.getTimeZone("UTC"));
        System.out.println("Time spent: " + df.format(new Date(millis)));
    }

    protected static SolrPostTool parseArgsAndInit(final String[] args) {
        String urlStr = null;
        try {
            final String mode = System.getProperty("data", "files");
            if (!SolrPostTool.DATA_MODES.contains(mode)) {
                fatal("System Property 'data' is not valid for this tool: " + mode);
            }
            final String params = System.getProperty("params", "");
            final String host = System.getProperty("host", "localhost");
            final String port = System.getProperty("port", "8983");
            final String core = System.getProperty("c");
            urlStr = System.getProperty("url");
            if (urlStr == null && core == null) {
                fatal("Specifying either url or core/collection is mandatory.\nUsage: java [SystemProperties] -jar post.jar [-h|-] [<file|folder|url|arg> [<file|folder|url|arg>...]]");
            }
            if (urlStr == null) {
                urlStr = String.format(Locale.ROOT, "http://%s:%s/solr/%s/update", host, port, core);
            }
            urlStr = appendParam(urlStr, params);
            final URL url = new URL(urlStr);
            String user = null;
            if (url.getUserInfo() != null && url.getUserInfo().trim().length() > 0) {
                user = url.getUserInfo().split(":")[0];
            } else if (System.getProperty("basicauth") != null) {
                user = System.getProperty("basicauth").trim().split(":")[0];
            }
            if (user != null) {
                info("Basic Authentication enabled, user=" + user);
            }
            final boolean auto = isOn(System.getProperty("auto", "no"));
            final String type = System.getProperty("type");
            final String format = System.getProperty("format");
            int recursive = 0;
            final String r = System.getProperty("recursive", "0");
            try {
                recursive = Integer.parseInt(r);
            } catch (Exception e) {
                if (isOn(r)) {
                    recursive = ("web".equals(mode) ? 1 : 999);
                }
            }
            int delay = "web".equals(mode) ? 10 : 0;
            try {
                delay = Integer.parseInt(System.getProperty("delay", "" + delay));
            } catch (Exception ex) {
            }
            final OutputStream out = isOn(System.getProperty("out", "no")) ? System.out : null;
            final String fileTypes = System.getProperty("filetypes", "xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log");
            final boolean commit = isOn(System.getProperty("commit", "yes"));
            final boolean optimize = isOn(System.getProperty("optimize", "no"));
            return new SolrPostTool(mode, url, auto, type, format, recursive, delay, fileTypes, out, commit, optimize, args);
        } catch (MalformedURLException e2) {
            fatal("System Property 'url' is not a valid URL: " + urlStr);
            return null;
        }
    }

    public SolrPostTool(final String mode, final URL url, final boolean auto, final String type, final String format, final int recursive, final int delay, final String fileTypes, final OutputStream out, final boolean commit, final boolean optimize, final String[] args) {
        this.auto = false;
        this.recursive = 0;
        this.delay = 0;
        this.out = null;
        this.backlog = new ArrayList<LinkedHashSet<URL>>();
        this.visited = new HashSet<URL>();
        this.mockMode = false;
        this.mode = mode;
        this.solrUrl = url;
        this.auto = auto;
        this.type = type;
        this.format = format;
        this.recursive = recursive;
        this.delay = delay;
        this.fileTypes = fileTypes;
        this.fileFilter = this.getFileFilterFromFileTypes(fileTypes);
        this.out = out;
        this.commit = commit;
        this.optimize = optimize;
        this.args = args;
        this.pageFetcher = new PageFetcher();
    }

    public SolrPostTool() {
        this.auto = false;
        this.recursive = 0;
        this.delay = 0;
        this.out = null;
        this.backlog = new ArrayList<LinkedHashSet<URL>>();
        this.visited = new HashSet<URL>();
        this.mockMode = false;
    }

    private void doFilesMode() {
        this.currentDepth = 0;
        if (!this.args[0].equals("-")) {
            info("Posting files to [base] url " + this.solrUrl + (this.auto ? "" : (" using content-type " + ((this.type == null) ? "application/xml" : this.type))) + "...");
            if (this.auto) {
                info("Entering auto mode. File endings considered are " + this.fileTypes);
            }
            if (this.recursive > 0) {
                info("Entering recursive mode, max depth=" + this.recursive + ", delay=" + this.delay + "s");
            }
            final int numFilesPosted = this.postFiles(this.args, 0, this.out, this.type);
            info(numFilesPosted + " files indexed.");
        }
    }

    private void doArgsMode() {
        info("POSTing args to " + this.solrUrl + "...");
        for (final String a : this.args) {
            this.postData(stringToStream(a), null, this.out, this.type, this.solrUrl);
        }
    }

    private int doWebMode() {
        this.reset();
        int numPagesPosted = 0;
        try {
            if (this.type != null) {
                fatal("Specifying content-type with \"-Ddata=web\" is not supported");
            }
            if (this.args[0].equals("-")) {
                return 0;
            }
            this.solrUrl = appendUrlPath(this.solrUrl, "/extract");
            info("Posting web pages to Solr url " + this.solrUrl);
            this.auto = true;
            info("Entering auto mode. Indexing pages with content-types corresponding to file endings " + this.fileTypes);
            if (this.recursive > 0) {
                if (this.recursive > 10) {
                    this.recursive = 10;
                    warn("Too large recursion depth for web mode, limiting to 10...");
                }
                if (this.delay < 10) {
                    warn("Never crawl an external web site faster than every 10 seconds, your IP will probably be blocked");
                }
                info("Entering recursive mode, depth=" + this.recursive + ", delay=" + this.delay + "s");
            }
            numPagesPosted = this.postWebPages(this.args, 0, this.out);
            info(numPagesPosted + " web pages indexed.");
        } catch (MalformedURLException e) {
            fatal("Wrong URL trying to append /extract to " + this.solrUrl);
        }
        return numPagesPosted;
    }

    private void doStdinMode() {
        info("POSTing stdin to " + this.solrUrl + "...");
        this.postData(System.in, null, this.out, this.type, this.solrUrl);
    }

    private void reset() {
        this.backlog = new ArrayList<LinkedHashSet<URL>>();
        this.visited = new HashSet<URL>();
    }

    private static void usageShort() {
        System.out.println("Usage: java [SystemProperties] -jar post.jar [-h|-] [<file|folder|url|arg> [<file|folder|url|arg>...]]\n       Please invoke with -h option for extended usage help.");
    }

    private static void usage() {
        System.out.println("Usage: java [SystemProperties] -jar post.jar [-h|-] [<file|folder|url|arg> [<file|folder|url|arg>...]]\n\nSupported System Properties and their defaults:\n  -Dc=<core/collection>\n  -Durl=<base Solr update URL> (overrides -Dc option if specified)\n  -Ddata=files|web|args|stdin (default=files)\n  -Dtype=<content-type> (default=application/xml)\n  -Dhost=<host> (default: localhost)\n  -Dport=<port> (default: 8983)\n  -Dbasicauth=<user:pass> (sets Basic Authentication credentials)\n  -Dauto=yes|no (default=no)\n  -Drecursive=yes|no|<depth> (default=0)\n  -Ddelay=<seconds> (default=0 for files, 10 for web)\n  -Dfiletypes=<type>[,<type>,...] (default=xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log)\n  -Dparams=\"<key>=<value>[&<key>=<value>...]\" (values must be URL-encoded)\n  -Dcommit=yes|no (default=yes)\n  -Doptimize=yes|no (default=no)\n  -Dout=yes|no (default=no)\n\nThis is a simple command line tool for POSTing raw data to a Solr port.\nNOTE: Specifying the url/core/collection name is mandatory.\nData can be read from files specified as commandline args,\nURLs specified as args, as raw commandline arg strings or via STDIN.\nExamples:\n  java -Dc=gettingstarted -jar post.jar *.xml\n  java -Ddata=args -Dc=gettingstarted -jar post.jar '<delete><id>42</id></delete>'\n  java -Ddata=stdin -Dc=gettingstarted -jar post.jar < hd.xml\n  java -Ddata=web -Dc=gettingstarted -jar post.jar http://example.com/\n  java -Dtype=text/csv -Dc=gettingstarted -jar post.jar *.csv\n  java -Dtype=application/json -Dc=gettingstarted -jar post.jar *.json\n  java -Durl=http://localhost:8983/solr/techproducts/update/extract -Dparams=literal.id=pdf1 -jar post.jar solr-word.pdf\n  java -Dauto -Dc=gettingstarted -jar post.jar *\n  java -Dauto -Dc=gettingstarted -Drecursive -jar post.jar afolder\n  java -Dauto -Dc=gettingstarted -Dfiletypes=ppt,html -jar post.jar afolder\nThe options controlled by System Properties include the Solr\nURL to POST to, the Content-Type of the data, whether a commit\nor optimize should be executed, and whether the response should\nbe written to STDOUT. If auto=yes the tool will try to set type\nautomatically from file name. When posting rich documents the\nfile name will be propagated as \"resource.name\" and also used\nas \"literal.id\". You may override these or any other request parameter\nthrough the -Dparams property. To do a commit only, use \"-\" as argument.\nThe web mode is a simple crawler following links within domain, default delay=10s.");
    }

    public int postFiles(final String[] args, final int startIndexInArgs, final OutputStream out, final String type) {
        this.reset();
        int filesPosted = 0;
        for (int j = startIndexInArgs; j < args.length; ++j) {
            final File srcFile = new File(args[j]);
            if (srcFile.isDirectory() && srcFile.canRead()) {
                filesPosted += this.postDirectory(srcFile, out, type);
            } else if (srcFile.isFile() && srcFile.canRead()) {
                filesPosted += this.postFiles(new File[]{srcFile}, out, type);
            } else {
                File parent = srcFile.getParentFile();
                if (parent == null) {
                    parent = new File(".");
                }
                final String fileGlob = srcFile.getName();
                final GlobFileFilter ff = new GlobFileFilter(fileGlob, false);
                final File[] files = parent.listFiles(ff);
                if (files == null || files.length == 0) {
                    warn("No files or directories matching " + srcFile);
                } else {
                    filesPosted += this.postFiles(parent.listFiles(ff), out, type);
                }
            }
        }
        return filesPosted;
    }

    public int postFiles(final File[] files, final int startIndexInArgs, final OutputStream out, final String type) {
        this.reset();
        int filesPosted = 0;
        for (final File srcFile : files) {
            if (srcFile.isDirectory() && srcFile.canRead()) {
                filesPosted += this.postDirectory(srcFile, out, type);
            } else if (srcFile.isFile() && srcFile.canRead()) {
                filesPosted += this.postFiles(new File[]{srcFile}, out, type);
            } else {
                File parent = srcFile.getParentFile();
                if (parent == null) {
                    parent = new File(".");
                }
                final String fileGlob = srcFile.getName();
                final GlobFileFilter ff = new GlobFileFilter(fileGlob, false);
                final File[] fileList = parent.listFiles(ff);
                if (fileList == null || fileList.length == 0) {
                    warn("No files or directories matching " + srcFile);
                } else {
                    filesPosted += this.postFiles(fileList, out, type);
                }
            }
        }
        return filesPosted;
    }

    private int postDirectory(final File dir, final OutputStream out, final String type) {
        if (dir.isHidden() && !dir.getName().equals(".")) {
            return 0;
        }
        info("Indexing directory " + dir.getPath() + " (" + dir.listFiles(this.fileFilter).length + " files, depth=" + this.currentDepth + ")");
        int posted = 0;
        posted += this.postFiles(dir.listFiles(this.fileFilter), out, type);
        if (this.recursive > this.currentDepth) {
            for (final File d : dir.listFiles()) {
                if (d.isDirectory()) {
                    ++this.currentDepth;
                    posted += this.postDirectory(d, out, type);
                    --this.currentDepth;
                }
            }
        }
        return posted;
    }

    int postFiles(final File[] files, final OutputStream out, final String type) {
        int filesPosted = 0;
        for (final File srcFile : files) {
            try {
                if (srcFile.isFile() && !srcFile.isHidden()) {
                    this.postFile(srcFile, out, type);
                    Thread.sleep(this.delay * 1000);
                    ++filesPosted;
                }
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        }
        return filesPosted;
    }

    public int postWebPages(final String[] args, final int startIndexInArgs, final OutputStream out) {
        this.reset();
        final LinkedHashSet<URL> s = new LinkedHashSet<URL>();
        for (int j = startIndexInArgs; j < args.length; ++j) {
            try {
                final URL u = new URL(normalizeUrlEnding(args[j]));
                s.add(u);
            } catch (MalformedURLException e) {
                warn("Skipping malformed input URL: " + args[j]);
            }
        }
        this.backlog.add(s);
        return this.webCrawl(0, out);
    }

    protected static String normalizeUrlEnding(String link) {
        if (link.indexOf("#") > -1) {
            link = link.substring(0, link.indexOf("#"));
        }
        if (link.endsWith("?")) {
            link = link.substring(0, link.length() - 1);
        }
        if (link.endsWith("/")) {
            link = link.substring(0, link.length() - 1);
        }
        return link;
    }

    protected int webCrawl(final int level, final OutputStream out) {
        int numPages = 0;
        final LinkedHashSet<URL> stack = this.backlog.get(level);
        final int rawStackSize = stack.size();
        stack.removeAll(this.visited);
        final int stackSize = stack.size();
        final LinkedHashSet<URL> subStack = new LinkedHashSet<URL>();
        info("Entering crawl at level " + level + " (" + rawStackSize + " links total, " + stackSize + " new)");
        for (URL u : stack) {
            try {
                this.visited.add(u);
                final PageFetcherResult result = this.pageFetcher.readPageFromUrl(u);
                if (result.httpStatus == 200) {
                    u = ((result.redirectUrl != null) ? result.redirectUrl : u);
                    final URL postUrl = new URL(appendParam(this.solrUrl.toString(), "literal.id=" + URLEncoder.encode(u.toString(), "UTF-8") + "&literal.url=" + URLEncoder.encode(u.toString(), "UTF-8")));
                    final boolean success = this.postData(new ByteArrayInputStream(result.content.array(), result.content.arrayOffset(), result.content.limit()), null, out, result.contentType, postUrl);
                    if (success) {
                        info("POSTed web resource " + u + " (depth: " + level + ")");
                        Thread.sleep(this.delay * 1000);
                        ++numPages;
                        if (this.recursive <= level || !result.contentType.equals("text/html")) {
                            continue;
                        }
                        final Set<URL> children = this.pageFetcher.getLinksFromWebPage(u, new ByteArrayInputStream(result.content.array(), result.content.arrayOffset(), result.content.limit()), result.contentType, postUrl);
                        subStack.addAll(children);
                    } else {
                        warn("An error occurred while posting " + u);
                    }
                } else {
                    warn("The URL " + u + " returned a HTTP result status of " + result.httpStatus);
                }
            } catch (IOException e) {
                warn("Caught exception when trying to open connection to " + u + ": " + e.getMessage());
            } catch (InterruptedException e2) {
                throw new RuntimeException(e2);
            }
        }
        if (!subStack.isEmpty()) {
            this.backlog.add(subStack);
            numPages += this.webCrawl(level + 1, out);
        }
        return numPages;
    }

    public static ByteBuffer inputStreamToByteArray(final InputStream is) throws IOException {
        return inputStreamToByteArray(is, 2147483647L);
    }

    public static ByteBuffer inputStreamToByteArray(final InputStream is, final long maxSize) throws IOException {
        try (final BAOS bos = new BAOS()) {
            long sz = 0L;
            for (int next = is.read(); next > -1; next = is.read()) {
                if (++sz > maxSize) {
                    throw new BufferOverflowException();
                }
                bos.write(next);
            }
            bos.flush();
            return bos.getByteBuffer();
        }
    }

    protected String computeFullUrl(final URL baseUrl, String link) {
        if (link == null || link.length() == 0) {
            return null;
        }
        if (!link.startsWith("http")) {
            if (link.startsWith("/")) {
                link = baseUrl.getProtocol() + "://" + baseUrl.getAuthority() + link;
            } else {
                if (link.contains(":")) {
                    return null;
                }
                String path = baseUrl.getPath();
                if (!path.endsWith("/")) {
                    final int sep = path.lastIndexOf("/");
                    final String file = path.substring(sep + 1);
                    if (file.contains(".") || file.contains("?")) {
                        path = path.substring(0, sep);
                    }
                }
                link = baseUrl.getProtocol() + "://" + baseUrl.getAuthority() + path + "/" + link;
            }
        }
        link = normalizeUrlEnding(link);
        final String l = link.toLowerCase(Locale.ROOT);
        if (l.endsWith(".jpg") || l.endsWith(".jpeg") || l.endsWith(".png") || l.endsWith(".gif")) {
            return null;
        }
        return link;
    }

    protected boolean typeSupported(final String type) {
        for (final String key : SolrPostTool.mimeMap.keySet()) {
            if (SolrPostTool.mimeMap.get(key).equals(type) && this.fileTypes.contains(key)) {
                return true;
            }
        }
        return false;
    }

    protected static boolean isOn(final String property) {
        return "true,on,yes,1".indexOf(property) > -1;
    }

    static void warn(final String msg) {
        System.err.println("PostTool: WARNING: " + msg);
    }

    static void info(final String msg) {
        System.out.println(msg);
    }

    static void fatal(final String msg) {
        System.err.println("PostTool: FATAL: " + msg);
        System.exit(2);
    }

    public void commit() {
        info("COMMITting Solr index changes to " + this.solrUrl + "...");
        this.doGet(appendParam(this.solrUrl.toString(), "commit=true"));
    }

    public void optimize() {
        info("Performing an OPTIMIZE to " + this.solrUrl + "...");
        this.doGet(appendParam(this.solrUrl.toString(), "optimize=true"));
    }

    public static String appendParam(String url, final String param) {
        final String[] split;
        final String[] pa = split = param.split("&");
        for (final String p : split) {
            if (p.trim().length() != 0) {
                final String[] kv = p.split("=");
                if (kv.length == 2) {
                    url = url + ((url.indexOf(63) > 0) ? "&" : "?") + kv[0] + "=" + kv[1];
                } else {
                    warn("Skipping param " + p + " which is not on form key=value");
                }
            }
        }
        return url;
    }

    public void postFile(final File file, final OutputStream output, String type) {
        InputStream is = null;
        try {
            URL url = this.solrUrl;
            String suffix = "";
            if (this.auto) {
                if (type == null) {
                    type = guessType(file);
                }
                if (type.equals("application/json") && !"solr".equals(this.format)) {
                    suffix = "/json/docs";
                    final String urlStr = appendUrlPath(this.solrUrl, suffix).toString();
                    url = new URL(urlStr);
                } else if (!type.equals("application/xml") && !type.equals("text/csv")) {
                    if (!type.equals("application/json")) {
                        suffix = "/extract";
                        String urlStr = appendUrlPath(this.solrUrl, suffix).toString();
                        if (urlStr.indexOf("resource.name") == -1) {
                            urlStr = appendParam(urlStr, "resource.name=" + URLEncoder.encode(file.getAbsolutePath(), "UTF-8"));
                        }
                        if (urlStr.indexOf("literal.id") == -1) {
                            urlStr = appendParam(urlStr, "literal.id=" + URLEncoder.encode(file.getAbsolutePath(), "UTF-8"));
                        }
                        url = new URL(urlStr);
                    }
                }
            } else if (type == null) {
                type = "application/xml";
            }
            info("POSTing file " + file.getName() + (this.auto ? (" (" + type + ")") : "") + " to [base]" + suffix + (this.mockMode ? " MOCK!" : ""));
            is = new FileInputStream(file);
            this.postData(is, file.length(), output, type, url);
        } catch (IOException e) {
            e.printStackTrace();
            warn("Can't open/read file: " + file);
            try {
                if (is != null) {
                    is.close();
                }
            } catch (IOException ex) {
                fatal("IOException while closing file: " + ex);
            }
        } finally {
            try {
                if (is != null) {
                    is.close();
                }
            } catch (IOException e2) {
                fatal("IOException while closing file: " + e2);
            }
        }
    }

    protected static URL appendUrlPath(final URL url, final String append) throws MalformedURLException {
        return new URL(url.getProtocol() + "://" + url.getAuthority() + url.getPath() + append + ((url.getQuery() != null) ? ("?" + url.getQuery()) : ""));
    }

    protected static String guessType(final File file) {
        final String name = file.getName();
        final String suffix = name.substring(name.lastIndexOf(".") + 1);
        final String type = SolrPostTool.mimeMap.get(suffix.toLowerCase(Locale.ROOT));
        return (type != null) ? type : "application/octet-stream";
    }

    public void doGet(final String url) {
        try {
            this.doGet(new URL(url));
        } catch (MalformedURLException e) {
            warn("The specified URL " + url + " is not a valid URL. Please check");
        }
    }

    public void doGet(final URL url) {
        try {
            if (this.mockMode) {
                return;
            }
            final HttpURLConnection urlc = (HttpURLConnection) url.openConnection();
            basicAuth(urlc);
            urlc.connect();
            checkResponseCode(urlc);
        } catch (IOException e2) {
            warn("An error occurred getting data from " + url + ". Please check that Solr is running.");
        } catch (Exception e) {
            warn("An error occurred getting data from " + url + ". Message: " + e.getMessage());
        }
    }

    public boolean postData(final InputStream data, final Long length, final OutputStream output, String type, final URL url) {
        if (this.mockMode) {
            return true;
        }
        boolean success = true;
        if (type == null) {
            type = "application/xml";
        }
        HttpURLConnection urlc = null;
        try {
            try {
                urlc = (HttpURLConnection) url.openConnection();
                try {
                    urlc.setRequestMethod("POST");
                } catch (ProtocolException e) {
                    fatal("Shouldn't happen: HttpURLConnection doesn't support POST??" + e);
                }
                urlc.setDoOutput(true);
                urlc.setDoInput(true);
                urlc.setUseCaches(false);
                urlc.setAllowUserInteraction(false);
                urlc.setRequestProperty("Content-type", type);
                basicAuth(urlc);
                if (null != length) {
                    urlc.setFixedLengthStreamingMode(length);
                } else {
                    urlc.setChunkedStreamingMode(-1);
                }
                urlc.connect();
            } catch (IOException e2) {
                fatal("Connection error (is Solr running at " + this.solrUrl + " ?): " + e2);
                success = false;
            } catch (Exception e3) {
                fatal("POST failed with error " + e3.getMessage());
            }
            try (final OutputStream out = urlc.getOutputStream()) {
                pipe(data, out);
            } catch (IOException e2) {
                fatal("IOException while posting data: " + e2);
            }
            try {
                success &= checkResponseCode(urlc);
                try (final InputStream in = urlc.getInputStream()) {
                    pipe(in, output);
                }
            } catch (IOException e2) {
                warn("IOException while reading response: " + e2);
                success = false;
            } catch (GeneralSecurityException e4) {
                fatal("Looks like Solr is secured and would not let us in. Try with another user in '-u' parameter");
            }
        } finally {
            if (urlc != null) {
                urlc.disconnect();
            }
        }
        return success;
    }

    private static void basicAuth(final HttpURLConnection urlc) throws Exception {
        if (urlc.getURL().getUserInfo() != null) {
            final String encoding = Base64.getEncoder().encodeToString(urlc.getURL().getUserInfo().getBytes(StandardCharsets.US_ASCII));
            urlc.setRequestProperty("Authorization", "Basic " + encoding);
        } else if (System.getProperty("basicauth") != null) {
            final String basicauth = System.getProperty("basicauth").trim();
            if (!basicauth.contains(":")) {
                throw new Exception("System property 'basicauth' must be of format user:pass");
            }
            urlc.setRequestProperty("Authorization", "Basic " + Base64.getEncoder().encodeToString(basicauth.getBytes(StandardCharsets.UTF_8)));
        }
    }

    private static boolean checkResponseCode(final HttpURLConnection urlc) throws IOException, GeneralSecurityException {
        if (urlc.getResponseCode() < 400) {
            return true;
        }
        warn("Solr returned an error #" + urlc.getResponseCode() + " (" + urlc.getResponseMessage() + ") for url: " + urlc.getURL());
        Charset charset = StandardCharsets.ISO_8859_1;
        final String contentType = urlc.getContentType();
        if (contentType != null) {
            final int idx = contentType.toLowerCase(Locale.ROOT).indexOf("charset=");
            if (idx > 0) {
                charset = Charset.forName(contentType.substring(idx + "charset=".length()).trim());
            }
        }
        try (final InputStream errStream = urlc.getErrorStream()) {
            if (errStream != null) {
                final BufferedReader br = new BufferedReader(new InputStreamReader(errStream, charset));
                final StringBuilder response = new StringBuilder("Response: ");
                int ch;
                while ((ch = br.read()) != -1) {
                    response.append((char) ch);
                }
                warn(response.toString().trim());
            }
        }
        if (urlc.getResponseCode() == 401) {
            throw new GeneralSecurityException("Solr requires authentication (response 401). Please try again with '-u' option");
        }
        if (urlc.getResponseCode() == 403) {
            throw new GeneralSecurityException("You are not authorized to perform this action against Solr. (response 403)");
        }
        return false;
    }

    public static InputStream stringToStream(final String s) {
        return new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8));
    }

    private static void pipe(final InputStream source, final OutputStream dest) throws IOException {
        final byte[] buf = new byte[1024];
        int read = 0;
        while ((read = source.read(buf)) >= 0) {
            if (null != dest) {
                dest.write(buf, 0, read);
            }
        }
        if (null != dest) {
            dest.flush();
        }
    }

    public FileFilter getFileFilterFromFileTypes(final String fileTypes) {
        String glob;
        if (fileTypes.equals("*")) {
            glob = ".*";
        } else {
            glob = "^.*\\.(" + fileTypes.replace(",", "|") + ")$";
        }
        return new GlobFileFilter(glob, true);
    }

    public static NodeList getNodesFromXP(final Node n, final String xpath) throws XPathExpressionException {
        final XPathFactory factory = XPathFactory.newInstance();
        final XPath xp = factory.newXPath();
        final XPathExpression expr = xp.compile(xpath);
        return (NodeList) expr.evaluate(n, XPathConstants.NODESET);
    }

    public static String getXP(final Node n, final String xpath, final boolean concatAll) throws XPathExpressionException {
        final NodeList nodes = getNodesFromXP(n, xpath);
        final StringBuilder sb = new StringBuilder();
        if (nodes.getLength() > 0) {
            for (int i = 0; i < nodes.getLength(); ++i) {
                sb.append(nodes.item(i).getNodeValue() + " ");
                if (!concatAll) {
                    break;
                }
            }
            return sb.toString().trim();
        }
        return "";
    }

    public static Document makeDom(final byte[] in) throws SAXException, IOException, ParserConfigurationException {
        final InputStream is = new ByteArrayInputStream(in);
        final Document dom = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is);
        return dom;
    }

    static {
        (DATA_MODES = new HashSet<String>()).add("files");
        SolrPostTool.DATA_MODES.add("args");
        SolrPostTool.DATA_MODES.add("stdin");
        SolrPostTool.DATA_MODES.add("web");
        (SolrPostTool.mimeMap = new HashMap<String, String>()).put("xml", "application/xml");
        SolrPostTool.mimeMap.put("csv", "text/csv");
        SolrPostTool.mimeMap.put("json", "application/json");
        SolrPostTool.mimeMap.put("jsonl", "application/json");
        SolrPostTool.mimeMap.put("pdf", "application/pdf");
        SolrPostTool.mimeMap.put("rtf", "text/rtf");
        SolrPostTool.mimeMap.put("html", "text/html");
        SolrPostTool.mimeMap.put("htm", "text/html");
        SolrPostTool.mimeMap.put("doc", "application/msword");
        SolrPostTool.mimeMap.put("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document");
        SolrPostTool.mimeMap.put("ppt", "application/vnd.ms-powerpoint");
        SolrPostTool.mimeMap.put("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation");
        SolrPostTool.mimeMap.put("xls", "application/vnd.ms-excel");
        SolrPostTool.mimeMap.put("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
        SolrPostTool.mimeMap.put("odt", "application/vnd.oasis.opendocument.text");
        SolrPostTool.mimeMap.put("ott", "application/vnd.oasis.opendocument.text");
        SolrPostTool.mimeMap.put("odp", "application/vnd.oasis.opendocument.presentation");
        SolrPostTool.mimeMap.put("otp", "application/vnd.oasis.opendocument.presentation");
        SolrPostTool.mimeMap.put("ods", "application/vnd.oasis.opendocument.spreadsheet");
        SolrPostTool.mimeMap.put("ots", "application/vnd.oasis.opendocument.spreadsheet");
        SolrPostTool.mimeMap.put("txt", "text/plain");
        SolrPostTool.mimeMap.put("log", "text/plain");
    }

    public static class BAOS extends ByteArrayOutputStream {
        public ByteBuffer getByteBuffer() {
            return ByteBuffer.wrap(super.buf, 0, super.count);
        }
    }

    static class GlobFileFilter implements FileFilter {
        private String _pattern;
        private Pattern p;

        public GlobFileFilter(final String pattern, final boolean isRegex) {
            this._pattern = pattern;
            if (!isRegex) {
                this._pattern = this._pattern.replace("^", "\\^").replace("$", "\\$").replace(".", "\\.").replace("(", "\\(").replace(")", "\\)").replace("+", "\\+").replace("*", ".*").replace("?", ".");
                this._pattern = "^" + this._pattern + "$";
            }
            try {
                this.p = Pattern.compile(this._pattern, 2);
            } catch (PatternSyntaxException e) {
                SolrPostTool.fatal("Invalid type list " + pattern + ". " + e.getDescription());
            }
        }

        @Override
        public boolean accept(final File file) {
            return this.p.matcher(file.getName()).find();
        }
    }

    class PageFetcher {
        Map<String, List<String>> robotsCache;
        static final String DISALLOW = "Disallow:";

        public PageFetcher() {
            this.robotsCache = new HashMap<String, List<String>>();
        }

        public PageFetcherResult readPageFromUrl(URL u) {
            final PageFetcherResult res = new PageFetcherResult();
            try {
                if (this.isDisallowedByRobots(u)) {
                    SolrPostTool.warn("The URL " + u + " is disallowed by robots.txt and will not be crawled.");
                    res.httpStatus = 403;
                    SolrPostTool.this.visited.add(u);
                    return res;
                }
                res.httpStatus = 404;
                final HttpURLConnection conn = (HttpURLConnection) u.openConnection();
                conn.setRequestProperty("User-Agent", "PostTool-crawler/5.0.0 (http://lucene.apache.org/solr/)");
                conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
                conn.connect();
                res.httpStatus = conn.getResponseCode();
                if (!SolrPostTool.normalizeUrlEnding(conn.getURL().toString()).equals(SolrPostTool.normalizeUrlEnding(u.toString()))) {
                    SolrPostTool.info("The URL " + u + " caused a redirect to " + conn.getURL());
                    u = conn.getURL();
                    res.redirectUrl = u;
                    SolrPostTool.this.visited.add(u);
                }
                if (res.httpStatus == 200) {
                    final String rawContentType = conn.getContentType();
                    final String type = rawContentType.split(";")[0];
                    if (SolrPostTool.this.typeSupported(type) || "*".equals(SolrPostTool.this.fileTypes)) {
                        final String encoding = conn.getContentEncoding();
                        InputStream is;
                        if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
                            is = new GZIPInputStream(conn.getInputStream());
                        } else if (encoding != null && encoding.equalsIgnoreCase("deflate")) {
                            is = new InflaterInputStream(conn.getInputStream(), new Inflater(true));
                        } else {
                            is = conn.getInputStream();
                        }
                        res.content = SolrPostTool.inputStreamToByteArray(is);
                        is.close();
                    } else {
                        SolrPostTool.warn("Skipping URL with unsupported type " + type);
                        res.httpStatus = 415;
                    }
                }
            } catch (IOException e) {
                SolrPostTool.warn("IOException when reading page from url " + u + ": " + e.getMessage());
            }
            return res;
        }

        public boolean isDisallowedByRobots(final URL url) {
            final String host = url.getHost();
            final String strRobot = url.getProtocol() + "://" + host + "/robots.txt";
            List<String> disallows = this.robotsCache.get(host);
            if (disallows == null) {
                disallows = new ArrayList<String>();
                try {
                    final URL urlRobot = new URL(strRobot);
                    disallows = this.parseRobotsTxt(urlRobot.openStream());
                } catch (MalformedURLException e) {
                    return true;
                } catch (IOException ex) {
                }
            }
            this.robotsCache.put(host, disallows);
            final String strURL = url.getFile();
            for (final String path : disallows) {
                if (path.equals("/") || strURL.indexOf(path) == 0) {
                    return true;
                }
            }
            return false;
        }

        protected List<String> parseRobotsTxt(final InputStream is) throws IOException {
            final List<String> disallows = new ArrayList<String>();
            final BufferedReader r = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
            String l;
            while ((l = r.readLine()) != null) {
                final String[] arr = l.split("#");
                if (arr.length == 0) {
                    continue;
                }
                l = arr[0].trim();
                if (!l.startsWith("Disallow:")) {
                    continue;
                }
                l = l.substring("Disallow:".length()).trim();
                if (l.length() == 0) {
                    continue;
                }
                disallows.add(l);
            }
            is.close();
            return disallows;
        }

        protected Set<URL> getLinksFromWebPage(final URL u, final InputStream is, final String type, final URL postUrl) {
            final Set<URL> l = new HashSet<URL>();
            URL url = null;
            try {
                final ByteArrayOutputStream os = new ByteArrayOutputStream();
                final URL extractUrl = new URL(SolrPostTool.appendParam(postUrl.toString(), "extractOnly=true"));
                final boolean success = SolrPostTool.this.postData(is, null, os, type, extractUrl);
                if (success) {
                    Document d = SolrPostTool.makeDom(os.toByteArray());
                    final String innerXml = SolrPostTool.getXP(d, "/response/str/text()[1]", false);
                    d = SolrPostTool.makeDom(innerXml.getBytes(StandardCharsets.UTF_8));
                    final NodeList links = SolrPostTool.getNodesFromXP(d, "/html/body//a/@href");
                    for (int i = 0; i < links.getLength(); ++i) {
                        String link = links.item(i).getTextContent();
                        link = SolrPostTool.this.computeFullUrl(u, link);
                        if (link != null) {
                            url = new URL(link);
                            if (url.getAuthority() != null) {
                                if (url.getAuthority().equals(u.getAuthority())) {
                                    l.add(url);
                                }
                            }
                        }
                    }
                }
            } catch (MalformedURLException e3) {
                SolrPostTool.warn("Malformed URL " + url);
            } catch (IOException e) {
                SolrPostTool.warn("IOException opening URL " + url + ": " + e.getMessage());
            } catch (Exception e2) {
                throw new RuntimeException(e2);
            }
            return l;
        }
    }

    public static class PageFetcherResult {
        int httpStatus;
        String contentType;
        URL redirectUrl;
        ByteBuffer content;

        public PageFetcherResult() {
            this.httpStatus = 200;
            this.contentType = "text/html";
            this.redirectUrl = null;
        }
    }
}
